{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'sys' (built-in)>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "import imp\n",
    "imp.reload(sys)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from collections import defaultdict\n",
    "import scipy.sparse as ss\n",
    "import pickle\n",
    "import scipy.io as sio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('E:/csdn/week5/triplet_dataset_sub_song_merged.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "triplet_train_sum_df = train[['user','listen_count']].groupby('user').sum().reset_index()\n",
    "triplet_train_sum_df.rename(columns = {'listen_count':'total_listen_count'},inplace = True)\n",
    "train = pd.merge(train,triplet_train_sum_df)\n",
    "train['fractional_play_count'] = train['listen_count']/train['total_listen_count']\n",
    "del triplet_train_sum_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "      <th>total_listen_count</th>\n",
       "      <th>fractional_play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAFTRR12AF72A8D4D</td>\n",
       "      <td>1</td>\n",
       "      <td>Harder Better Faster Stronger</td>\n",
       "      <td>Discovery</td>\n",
       "      <td>Daft Punk</td>\n",
       "      <td>2007</td>\n",
       "      <td>861</td>\n",
       "      <td>0.001161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAIILB12A58A776F7</td>\n",
       "      <td>3</td>\n",
       "      <td>Phantom Part 1.5 (Album Version)</td>\n",
       "      <td>A Cross The Universe</td>\n",
       "      <td>Justice</td>\n",
       "      <td>0</td>\n",
       "      <td>861</td>\n",
       "      <td>0.003484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAJJDS12A8C13A3FB</td>\n",
       "      <td>1</td>\n",
       "      <td>I Got Mine</td>\n",
       "      <td>Attack &amp; Release</td>\n",
       "      <td>The Black Keys</td>\n",
       "      <td>2008</td>\n",
       "      <td>861</td>\n",
       "      <td>0.001161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMDXO12A8C131E2F</td>\n",
       "      <td>2</td>\n",
       "      <td>Pogo</td>\n",
       "      <td>Idealism</td>\n",
       "      <td>Digitalism</td>\n",
       "      <td>2007</td>\n",
       "      <td>861</td>\n",
       "      <td>0.002323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMPRJ12A8AE45F38</td>\n",
       "      <td>20</td>\n",
       "      <td>Rorol</td>\n",
       "      <td>Identification Parade</td>\n",
       "      <td>Octopus Project</td>\n",
       "      <td>2002</td>\n",
       "      <td>861</td>\n",
       "      <td>0.023229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  listen_count  \\\n",
       "0  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAFTRR12AF72A8D4D             1   \n",
       "1  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAIILB12A58A776F7             3   \n",
       "2  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAJJDS12A8C13A3FB             1   \n",
       "3  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMDXO12A8C131E2F             2   \n",
       "4  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMPRJ12A8AE45F38            20   \n",
       "\n",
       "                              title                release      artist_name  \\\n",
       "0     Harder Better Faster Stronger              Discovery        Daft Punk   \n",
       "1  Phantom Part 1.5 (Album Version)   A Cross The Universe          Justice   \n",
       "2                        I Got Mine       Attack & Release   The Black Keys   \n",
       "3                              Pogo               Idealism       Digitalism   \n",
       "4                             Rorol  Identification Parade  Octopus Project   \n",
       "\n",
       "   year  total_listen_count  fractional_play_count  \n",
       "0  2007                 861               0.001161  \n",
       "1     0                 861               0.003484  \n",
       "2  2008                 861               0.001161  \n",
       "3  2007                 861               0.002323  \n",
       "4  2002                 861               0.023229  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "子集抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "song_count_df = pd.read_csv('E:/csdn/week5/song_playcount_df.csv')\n",
    "song_count_subset = song_count_df.head(n=1000)\n",
    "song_subset = list(song_count_subset.song)\n",
    "train_sub = train[train.song.isin(song_subset)]\n",
    "\n",
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(211686, 9)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = list(song_count_subset['song'])\n",
    "users = list(train_sub['user'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users : 4974\n",
      "number of songs : 1000\n"
     ]
    }
   ],
   "source": [
    "n_users = len(users)\n",
    "n_items = len(items)\n",
    "print (\"number of users : %d\" % n_users)\n",
    "print (\"number of songs : %d\" % n_items)\n",
    "\n",
    "user_items = defaultdict(set)\n",
    "item_users = defaultdict(set)\n",
    "\n",
    "user_items_scores = ss.dok_matrix((n_users,n_items))\n",
    "\n",
    "user_index = dict()\n",
    "item_index = dict()\n",
    "for i,u in enumerate(users):\n",
    "    user_index[u] = i\n",
    "    \n",
    "    \n",
    "for i,e in enumerate(items):\n",
    "    item_index[e] = i\n",
    "    \n",
    "n_records = train_sub.shape[0]\n",
    "for i in range(n_records):\n",
    "    user_index_i = user_index[train_sub.iloc[i]['user']]\n",
    "    item_index_i = item_index[train_sub.iloc[i]['song']]\n",
    "    \n",
    "    user_items[user_index_i].add(item_index_i)\n",
    "    item_users[item_index_i].add(user_index_i)\n",
    "    \n",
    "    score = train_sub.iloc[i]['fractional_play_count']\n",
    "    user_items_scores[user_index_i,item_index_i] = score\n",
    "    \n",
    "\n",
    "pickle.dump(user_items, open(\"user_items.pkl\",'wb'))\n",
    "pickle.dump(item_users,open(\"item_users.pkl\",'wb'))\n",
    "\n",
    "sio.mmwrite(\"user_items_scores\",user_items_scores)\n",
    "\n",
    "pickle.dump(user_index,open(\"user_index.pkl\",'wb'))\n",
    "pickle.dump(item_index,open(\"item_index.pkl\",'wb'))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "计算相似性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i=:0\n",
      "i=:10\n",
      "i=:20\n",
      "i=:30\n",
      "i=:40\n",
      "i=:50\n",
      "i=:60\n",
      "i=:70\n",
      "i=:80\n",
      "i=:90\n",
      "i=:100\n",
      "i=:110\n",
      "i=:120\n",
      "i=:130\n",
      "i=:140\n",
      "i=:150\n",
      "i=:160\n",
      "i=:170\n",
      "i=:180\n",
      "i=:190\n",
      "i=:200\n",
      "i=:210\n",
      "i=:220\n",
      "i=:230\n",
      "i=:240\n",
      "i=:250\n",
      "i=:260\n",
      "i=:270\n",
      "i=:280\n",
      "i=:290\n",
      "i=:300\n",
      "i=:310\n",
      "i=:320\n",
      "i=:330\n",
      "i=:340\n",
      "i=:350\n",
      "i=:360\n",
      "i=:370\n",
      "i=:380\n",
      "i=:390\n",
      "i=:400\n",
      "i=:410\n",
      "i=:420\n",
      "i=:430\n",
      "i=:440\n",
      "i=:450\n",
      "i=:460\n",
      "i=:470\n",
      "i=:480\n",
      "i=:490\n",
      "i=:500\n",
      "i=:510\n",
      "i=:520\n",
      "i=:530\n",
      "i=:540\n",
      "i=:550\n",
      "i=:560\n",
      "i=:570\n",
      "i=:580\n",
      "i=:590\n",
      "i=:600\n",
      "i=:610\n",
      "i=:620\n",
      "i=:630\n",
      "i=:640\n",
      "i=:650\n",
      "i=:660\n",
      "i=:670\n",
      "i=:680\n",
      "i=:690\n",
      "i=:700\n",
      "i=:710\n",
      "i=:720\n",
      "i=:730\n",
      "i=:740\n",
      "i=:750\n",
      "i=:760\n",
      "i=:770\n",
      "i=:780\n",
      "i=:790\n",
      "i=:800\n",
      "i=:810\n",
      "i=:820\n",
      "i=:830\n",
      "i=:840\n",
      "i=:850\n",
      "i=:860\n",
      "i=:870\n",
      "i=:880\n",
      "i=:890\n",
      "i=:900\n",
      "i=:910\n",
      "i=:920\n",
      "i=:930\n",
      "i=:940\n",
      "i=:950\n",
      "i=:960\n",
      "i=:970\n",
      "i=:980\n",
      "i=:990\n"
     ]
    }
   ],
   "source": [
    "similarity_matrix = np.matrix(np.zeros(shape = (n_items,n_items)),float)\n",
    "\n",
    "for i in range(n_items):\n",
    "    users_i = item_users[i]\n",
    "    similarity_matrix[i,i] = 1.0\n",
    "    \n",
    "    if (i % 10 == 0 ):\n",
    "        print(\"i=:%d\" % (i))\n",
    "        for j in range(i+1,n_items):\n",
    "            users_j = item_users[j]\n",
    "            \n",
    "            users_intersection = users_i.intersection(users_j)\n",
    "            \n",
    "            if len(users_intersection)!= 0:\n",
    "                users_union = users_i.union(users_j)\n",
    "                similarity_matrix[j,i] = float(len(users_intersection))/float(len(users_union))\n",
    "            else:\n",
    "                 similarity_matrix[j,i] = 0\n",
    "        \n",
    "        similarity_matrix[i,j] = similarity_matrix[j,i]\n",
    "        pickle.dump(similarity_matrix,open(\"items_similarity.pkl\",'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
